import java.io.IOException;
import java.io.InputStream;

/**
 * A class that will tokenize a file
 * @author gabriel
 *
 */
public class WordTokenizer {

    /** The set of variable break characters */
    public static final String VARIABLE_BREAKS = " :/.,+*()~&%$#!`\"{}|^[]\\@?<>=;";
    
    /** The start of the low range of fixed breaks */
    public static final int FIXED_BREAK_LOW1 = 0;

    /** The end of the low range of fixed breaks; */
    public static final int FIXED_BREAK_UPPER1 = 31; // Octal 37

    /** The start of the high range of fixed breaks */
    public static final int FIXED_BREAK_LOW2 = 127; // Octal 177

    /** The end of the low range of fixed breaks */
    public static final int FIXED_BREAK_UPPER2 = 255; // Octal 377
    
    public static final int LINE_FEED = 10;
    public static final int CARRIAGE_RETURN = 13;
    /**
     * File stream to tokenize
     */
    private InputStream iStream;
    
    /**
     * Break table, that determines if each character is or is not a break;
     */
    private boolean[] asciiBreakTable;
    
    /**
     * Tells us if building the break table succeeded
     */
    private boolean breakTableSuccess = true;
    
    /**
     * Indicates any override break character that is erroneous
     */
    private String errorMessage = "";

    //used for keeping track of the previous character, needed by determining line feeds
    //that are preceded by carriage returns
    private int lastch;

    //used for keeping track of state to obtain file statistics and to get line numbers for
    //the occurrence of a phrase
    private int currLineNumber;
    private int numWords;
    private int numChars;
    private int numLines;

    //variable used for detecting a Carriage return-line feed sequence
    private boolean justFinishedLine;
    
    /**
     * Initializes the tokenizer. It will build a breaktable, whose validity
     * should be tested using breaksAreValid, in case of invalid user input
     * @param inputStream The input stream to tokenize
     * @param overriddenBreaks The variable breaks to override Should be null
     *      if we are not going to overide the defaults.
     */
    public WordTokenizer(InputStream inputStream, String overriddenBreaks) {
        asciiBreakTable = this.buildBreakTable(overriddenBreaks);
        iStream = inputStream;
        numWords=0;
        numChars=0;
        numLines=1;
        currLineNumber=1;
        lastch=-1;
    }

    /**
     * returns the current line in the file
     * @return
     */
    public int getCurrLineNumber() {
    	if (justFinishedLine) {
    		return (currLineNumber - 1);
    	}
    	return currLineNumber;
    }
    
    /**
     * returns the number of lines read so far
     * @return
     */
    public int getNumLines()
    {
    	return numLines;
    }
    /**
     * returns the number of characters read so far
     * @return
     */
    public int getNumChars()
    {
    	return numChars;
    }
    /**
     * returns the number of words read so far
     * @return
     */
    public int getNumWords()
    {
    	return numWords;
    }
    
    /**
     * Tokenizes strings from an input stream according to the break characters
     * specified and default
     * @return The next token in the stream
     * @throws IOException If file could not be read
     */
    public String nextToken() throws IOException {
        int ch;
        String retString = "";
        boolean isTokenState = false;
        while ((ch = iStream.read()) != -1) {
        	numChars++;
            boolean charIsBreak = asciiBreakTable[ch];
            //determine whether we hit a newline
            //ignore a line feed if it is preceded by a carriage return
            if(ch==CARRIAGE_RETURN || ch==LINE_FEED)
            {
            	justFinishedLine = true;
            	currLineNumber++;
            	numLines++;
            	if (ch==LINE_FEED && lastch==CARRIAGE_RETURN) {
                	currLineNumber--;
                	numLines--;
            	}
            } else {
            	justFinishedLine = false;
            }
            lastch=ch;
            if (isTokenState) {
                /* See if the word ends with a break, if so, increment count */
                if (charIsBreak) {
                    isTokenState = false;
                    numWords++;
                    return retString;
                } else {
                    retString += (char) ch;
                }
            } else {
                /*
                * We are in the initial state (reading no characters or
                * reading breaks. If we get a non break, we get into the
                * token state.
                */
                if (!charIsBreak) {
                    retString += (char) ch;
                    isTokenState = true;
                }
            }
        }
        /* Return the last word if we encountered an eof before hitting a
            * break
            */
        if (isTokenState) {
        	numWords++;
            return retString;
        }
        return null;
    }

    /**
     * Tests if a character is a fixed break
     * @param c the character, in ascii to test
     * @return True if the character is a fixed break
     */
    private boolean isFixedBreak(int c) {
        return ((c >= WordTokenizer.FIXED_BREAK_LOW1) &&
                (c <= WordTokenizer.FIXED_BREAK_UPPER1)) ||
                ((c >= WordTokenizer.FIXED_BREAK_LOW2) &&
                 (c <= WordTokenizer.FIXED_BREAK_UPPER2));
    }

    /**
    * Builds a table of breaks, handling the override breaks. 
    * @param overriddenBreaks The set of breaks to override
    * @return A boolean array where each character value is an index
    *  and each index is true if and only if the character is a break
    *  null is returned if there is an error with the overridden breaks.
    */
    private boolean[] buildBreakTable(String overriddenBreaks) {
        int ch;
        boolean[] breakTable = new boolean[256];
        byte[] temp1 = WordTokenizer.VARIABLE_BREAKS.getBytes();
        for (int i = 0; i < breakTable.length; i++) {
            breakTable[i] = false;
        }
        /* Add the fixed breaks - lowerrange*/
        for (int i = FIXED_BREAK_LOW1; i <= FIXED_BREAK_UPPER1; i++) {
            breakTable[i] = true;
        }
        /* Upper range */
        for (int i = FIXED_BREAK_LOW2; i <= FIXED_BREAK_UPPER2; i++) {
            breakTable[i] = true;
        }
        if (overriddenBreaks == null) {
            /* Set the variable breaks */
            for (int i = 0; i < temp1.length; i++) {
                breakTable[temp1[i]] = true;
            }
        } else {
            temp1 = overriddenBreaks.getBytes();
            /*
             * This loop will verify user input overridden variable breaks. It will
             * ensure that each character is in the set of variable breaks and that
             * it is not duplicated.
             */
            for (int i = 0; i < temp1.length; i++) {
                ch = temp1[i];
                if (breakTable[ch]) {
                    /*
                     * This catches Duplicates and characters that are already
                     * fixed breaks
                     */
                     this.breakTableSuccess = false;
                     if (this.isFixedBreak(ch)) {
                         this.errorMessage = "Character " + (char) ch +
                             " is already a fixed break.";
                     } else {
                         this.errorMessage = "Character " + (char) ch +
                             " is specified more than once as a variable break";
                     }
                     break;
                    
                } else {
                    breakTable[ch] = true;
                }
            }
                
        }
        return breakTable;
    }
    
    /**
     * Checks if building the breaktable was a success (i.e. no invalid input)
     * @return True if the breaktable was created successfully
     */
    public boolean breaksAreValid() {
        return this.breakTableSuccess;
    }

    /**
     * @return Returns the errorMessage.
     */
    public String getErrorMessage() {
        return errorMessage;
    }
}
